#ifndef ASM_BARRIER_H
#define ASM_BARRIER_H

#ifndef __ASSEMBLY__

#define __nops(n)	".rept	" #n "\nnop\n.endr\n"
#define nops(n)		asm volatile(__nops(n))

#define sev()		asm volatile("sev" : : : "memory")
#define wfe()		asm volatile("wfe" : : : "memory")
#define wfet(val)	asm volatile("msr s0_3_c1_c0_0, %0"	\
                     : : "r" (val) : "memory")

#define wfi()		asm volatile("wfi" : : : "memory")
#define wfit(val)	asm volatile("msr s0_3_c1_c0_1, %0"	\
                     : : "r" (val) : "memory")

#define isb()		asm volatile("isb" : : : "memory")
#define dmb(opt)	asm volatile("dmb " #opt : : : "memory")
#define dsb(opt)	asm volatile("dsb " #opt : : : "memory")

#define psb_csync()	asm volatile("hint #17" : : : "memory")
#define csdb()		asm volatile("hint #20" : : : "memory")

/*
 * Data Gathering Hint:
 * This instruction prevents merging memory accesses with Normal-NC or
 * Device-GRE attributes before the hint instruction with any memory accesses
 * appearing after the hint instruction.
 */
#define dgh()		asm volatile("hint #6" : : : "memory")

#define spec_bar()	asm volatile("dsb nsh\nisb\n")

#define __mb()		dsb(sy)
#define __rmb()		dsb(ld)
#define __wmb()		dsb(st)

#define __dma_mb()	dmb(osh)
#define __dma_rmb()	dmb(oshld)
#define __dma_wmb()	dmb(oshst)

#define io_stop_wc()	dgh()

/*
 * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz
 * and 0 otherwise.
 */
#define array_index_mask_nospec array_index_mask_nospec
static inline unsigned long array_index_mask_nospec(unsigned long idx,
                            unsigned long sz)
{
    unsigned long mask;

    asm volatile(
    "	cmp	%1, %2\n"
    "	sbc	%0, xzr, xzr\n"
    : "=r" (mask)
    : "r" (idx), "Ir" (sz)
    : "cc");

    csdb();
    return mask;
}

/*
 * Ensure that reads of the counter are treated the same as memory reads
 * for the purposes of ordering by subsequent memory barriers.
 *
 * This insanity brought to you by speculative system register reads,
 * out-of-order memory accesses, sequence locks and Thomas Gleixner.
 *
 * https://lore.kernel.org/r/alpine.DEB.2.21.1902081950260.1662@nanos.tec.linutronix.de/
 */
#define arch_counter_enforce_ordering(val) do {				\
    unsigned long tmp, _val = (val);						\
                                    \
    asm volatile(							\
    "	eor	%0, %1, %1\n"					\
    "	add	%0, sp, %0\n"					\
    "	ldr	xzr, [%0]"					\
    : "=r" (tmp) : "r" (_val));					\
} while (0)

#define __smp_mb()	dmb(ish)
#define __smp_rmb()	dmb(ishld)
#define __smp_wmb()	dmb(ishst)

#define __smp_store_release(p, v)					\
do {									\
    typeof(p) __p = (p);						\
    union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u =	\
        { .__val = (__unqual_scalar_typeof(*p)) (v) };	\
    switch (sizeof(*p)) {						\
    case 1:								\
        asm volatile ("stlrb %w1, %0"				\
                : "=Q" (*__p)				\
                : "r" (*(__u8 *)__u.__c)		\
                : "memory");				\
        break;							\
    case 2:								\
        asm volatile ("stlrh %w1, %0"				\
                : "=Q" (*__p)				\
                : "r" (*(__u16 *)__u.__c)		\
                : "memory");				\
        break;							\
    case 4:								\
        asm volatile ("stlr %w1, %0"				\
                : "=Q" (*__p)				\
                : "r" (*(__u32 *)__u.__c)		\
                : "memory");				\
        break;							\
    case 8:								\
        asm volatile ("stlr %1, %0"				\
                : "=Q" (*__p)				\
                : "r" (*(__u64 *)__u.__c)		\
                : "memory");				\
        break;							\
    }								\
} while (0)

#define __smp_load_acquire(p)						\
({									\
    union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u;	\
    typeof(p) __p = (p);						\
    switch (sizeof(*p)) {						\
    case 1:								\
        asm volatile ("ldarb %w0, %1"				\
            : "=r" (*(__u8 *)__u.__c)			\
            : "Q" (*__p) : "memory");			\
        break;							\
    case 2:								\
        asm volatile ("ldarh %w0, %1"				\
            : "=r" (*(__u16 *)__u.__c)			\
            : "Q" (*__p) : "memory");			\
        break;							\
    case 4:								\
        asm volatile ("ldar %w0, %1"				\
            : "=r" (*(__u32 *)__u.__c)			\
            : "Q" (*__p) : "memory");			\
        break;							\
    case 8:								\
        asm volatile ("ldar %0, %1"				\
            : "=r" (*(__u64 *)__u.__c)			\
            : "Q" (*__p) : "memory");			\
        break;							\
    }								\
    (typeof(*p))__u.__val;						\
})

#define smp_cond_load_relaxed(ptr, cond_expr)				\
({									\
    typeof(ptr) __PTR = (ptr);					\
    __unqual_scalar_typeof(*ptr) VAL;				\
    for (;;) {							\
        VAL = READ_ONCE(*__PTR);				\
        if (cond_expr)						\
            break;						\
        __cmpwait_relaxed(__PTR, VAL);				\
    }								\
    (typeof(*ptr))VAL;						\
})

#define smp_cond_load_acquire(ptr, cond_expr)				\
({									\
    typeof(ptr) __PTR = (ptr);					\
    __unqual_scalar_typeof(*ptr) VAL;				\
    for (;;) {							\
        VAL = smp_load_acquire(__PTR);				\
        if (cond_expr)						\
            break;						\
        __cmpwait_relaxed(__PTR, VAL);				\
    }								\
    (typeof(*ptr))VAL;						\
})

#include <asm-generic/barrier.h>

#endif /* !__ASSEMBLY__ */
#endif /* !ASM_BARRIER_H */
